package com.guiji.quartz.domain;

import cn.hutool.core.util.RandomUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.config.ProxyConfig;
import com.guiji.quartz.task.DataSaveUtil;
import com.guiji.quartz.util.DateFormatUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Techcrunch {
	private static final Logger log = LoggerFactory.getLogger(Techcrunch.class);

	public static void mainMethod() throws Exception {
		List<String> list = new ArrayList<>();
		list.add("https://techcrunch.com/");
		list.add("https://techcrunch.com/asia/");
		list.add("https://techcrunch.com/tag/venture-capital/");
		list.forEach(url ->{
			HttpResponse response = null;
			try {
				response = HttpRequest.get(url)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
					.header("Accept-Encoding", "gzip, deflate")
					.setHttpProxy(ProxyConfig.url, ProxyConfig.ip)
					.setConnectionTimeout(15000)
					.execute();
			} catch (Exception e) {
				log.error("代理失败1" + e);
				return;
			}
			String category = "Techcrunch-IndiaTech";
			if(url.contains("asia")){
				category = "Techcrunch-AsianTech";
			}
			if(url.contains("venture")){
				category = "Techcrunch-VentueCapital";
			}
			String resp = response.body();
			List<String> resUrl = getUrl(resp);
			getContent(resUrl,category);
		});

	}

	private static void getContent(List<String> listContent,String category) {
		for (String url : listContent) {
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc;
			JXDocument jxDocument;
			try {
				HttpResponse response = HttpRequest.get(url).setHttpProxy(ProxyConfig.url, ProxyConfig.ip).setConnectionTimeout(15000).execute();
				doc = response.body();
				jxDocument = new JXDocument(doc);
			} catch (Exception e) {
				log.info("代理失败2" + e);
				continue;
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//h1/text()").get(0);
			} catch (Exception e) {
				log.error("标题错误-------------" + url + "-------------------标题错误");
				continue;
			}
			Document document = null;
			String content = null;
			try {
				document = Jsoup.parse(doc);
				content = document.getElementsByClass("article-content").get(0).toString();
				content = content.replaceAll("\n", "\\$\\$\\$").replaceAll("<style.*?</style>|<article.*?</article>|<iframe.*?</iframe>|<iframe.*?</iframe>", " ").replaceAll("<.*?>", " ");
			} catch (Exception e) {
				continue;
			}

			List<String> newImgs = new ArrayList<>();

			Date date = null;
			try {
				date = DateFormatUtil.format(document.getElementsByClass("time-red").get(0).toString().replaceAll("<.*?>|,", ""));
			} catch (Exception e) {
				log.error("获取时间失败");
				date = new Date();
			}


			if (content.length() > 50000 || content.toString().replaceAll("\\$|[\\s\\p{Zs}]","").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				String categoryTemp = category;
				if(category.equals("Techcrunch-IndiaTech")){
					int i = RandomUtil.randomInt(1, 3);
					categoryTemp = category+"-"+i;
				}
				DataSaveUtil.saveData(title, content, newImgs, url, date, "https://techcrunch.com/||", categoryTemp);
				log.info(Thread.currentThread().getName() + "-----------------Techcrunch----------------");
			} catch (Exception e) {
				continue;
			}

		}
	}

	private static List<String> getUrl(String content) {
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String s = sdf.format(new Date());
		String[] ss = sdf.format(new Date()).split("-");
		String year = ss[0];
		String mon = ss[1];
		String day = ss[2];//https://techcrunch.com/2021/11/08/amazon-is-adding-a-venmo-checkout-option-next-year/

		List<String> res = new ArrayList<>();
		String pattern = "/"+year+"/"+mon+"/[0-9]{2}/.*?/";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		while (m.find()) {
			res.add("https://techcrunch.com" + m.group());
		}
		return res;
	}

//	public static void main(String[] args) throws Exception {
//		mainMethod();
//	}
}

